{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "import tensorflow.keras.layers as layers\n",
    "tf.keras.backend.clear_session()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "inputs = tf.keras.Input(shape=(784,),name='img')\n",
    "h1 = layers.Dense(32,activation='relu')(inputs)\n",
    "h2 = layers.Dense(32,activation='relu')(h1)\n",
    "outputs = layers.Dense(10,activation='softmax')(h2)\n",
    "model = tf.keras.Model(inputs=inputs,outputs=outputs,name='mnist_model')\n",
    "\n",
    "model.summary()\n",
    "keras.utils.plot_model(model,'output/mnist_model.png')\n",
    "keras.utils.plot_model(model,'output/model_info.png',show_shapes=True)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "(x_train,y_train),(x_test,y_test) = keras.datasets.mnist.load_data()\n",
    "x_train = x_train.reshape(60000,784).astype('float32')/255\n",
    "x_test = x_test.reshape(10000,784).astype('float32')/255\n",
    "model.compile(optimizer=keras.optimizers.RMSprop(),\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "history = model.fit(x_train,y_train,batch_size=64,epochs=5,validation_split=0.2)\n",
    "test_scores=model.evaluate(x_test,y_test,verbose=0)\n",
    "print('test loss:',test_scores[0])\n",
    "print('test acc:',test_scores[1])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model.save('output/model_save.h5')\n",
    "del model\n",
    "model = keras.models.load_model(\"output/model_save.h5\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "encode_input = keras.Input(shape=(28,29,1),name='img')\n",
    "h1 = layers.Conv2D(16,3,activation='relu')(encode_input)\n",
    "h1 = layers.Conv2D(32,2,activation='relu')(h1)\n",
    "h1 = layers.MaxPool2D(3)(h1)\n",
    "h1 = layers.Conv2D(32,3,activation='relu')(h1)\n",
    "h1 = layers.Conv2D(16,3,activation='relu')(h1)\n",
    "encode_output = layers.GlobalMaxPool2D()(h1)\n",
    "\n",
    "encode_model = keras.Model(inputs=encode_input,outputs=encode_output,name='encoder')\n",
    "encode_model.summary()\n",
    "\n",
    "h2 = layers.Reshape((4,4,1))(encode_output)\n",
    "h2 = layers.Conv2DTranspose(16,3,activation='relu')(h2)\n",
    "h2 = layers.Conv2DTranspose(32,3,activation='relu')(h2)\n",
    "h2 = layers.UpSampling2D(3)(h2)\n",
    "h2 = layers.Conv2DTranspose(16,3,activation='relu')(h2)\n",
    "decode_output = layers.Conv2DTranspose(1,3,activation='relu')(h2)\n",
    "\n",
    "autoencoder = keras.Model(inptus =encode_output,outputs=decode_output,name='autoencoder')\n",
    "autoencoder.summary()\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "encode_input = keras.Input(shape=(28,28,1),name='src_img')\n",
    "h1 = layers.Conv2D(16,3,activation='relu')(encode_input)\n",
    "h1 = layers.Conv2D(32,3,activation='relu')(h1)\n",
    "h1 = layers.MaxPool2D(3)(h1)\n",
    "h1 = layers.Conv2D(32,3,activation='relu')(h1)\n",
    "h1 = layers.Conv2D(16,3,activation='relu')(h1)\n",
    "encode_output = layers.GlobalMaxPool2D()(h1)\n",
    "\n",
    "encode_model = keras.Model(inptus=encode_input,outputs=encode_output,name='encoder')\n",
    "encode_model.summary()\n",
    "\n",
    "decode_input = keras.Input(shape=(16,),name='encoded_img')\n",
    "h2 = layers.Reshape((4,4,1))(decode_input)\n",
    "h2 = layers.Conv2DTranspose(16,3,activation='relu')(h2)\n",
    "h2 = layers.UpSampling2D(3)(h2)\n",
    "h2 = layers.Conv2DTranspose(16,3,activation='relu')(h2)\n",
    "decode_output = layers.Conv2DTranspose(1,3,activation='relu')(h2)\n",
    "decode_model = keras.Model(inputs=decode_input,outputs=decode_output,name='decoder')\n",
    "decode_model.summary()\n",
    "\n",
    "autoencoder_input = keras.Input(shape=(28,28,1),name='img')\n",
    "h3 = encode_model(autoencoder_input)\n",
    "autoencoder_output =decode_model(h3)\n",
    "autoencoder = keras.Model(inputs=autoencoder_input,outputs=autoencoder_output,\n",
    "                          name='autoencoder')\n",
    "autoencoder.summary()\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    inputs = keras.Input(shape=(128,))\n",
    "    outputs = keras.layers.Dense(1,activation='sigmoid')(inputs)\n",
    "    return keras.Model(inputs,outputs)\n",
    "\n",
    "model1 = get_model()\n",
    "model2 = get_model()\n",
    "model3 = get_model()\n",
    "inputs = keras.Input(shape=(128,))\n",
    "y1 = model1(inputs)\n",
    "y2 = model2(inputs)\n",
    "y3 = model3(inputs)\n",
    "\n",
    "outputs = layers.average([y1,y2,y3])\n",
    "ensemble_model = keras.Model(inputs,outputs)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "num_words = 2000\n",
    "num_tags = 12\n",
    "num_departments= 4\n",
    "\n",
    "body_input = keras.Input(shape=(None,),name='body')\n",
    "title_input = keras.Input(shape=(None,),name='title')\n",
    "tag_input = keras.Input(shape=(num_tags,),name='tag')\n",
    "\n",
    "body_feat = layers.Embedding(num_words,64)(body_input)\n",
    "title_feat = layers.Embedding(num_words,64)(title_input)\n",
    "\n",
    "body_feat = layers.LSTM(32)(body_feat)\n",
    "title_feat = layers.LSTM(128)(title_feat)\n",
    "features = layers.concatenate([title_feat,body_feat,tag_input])\n",
    "\n",
    "priority_pred = layers.Dense(1,activation='sigmoid',name='priority')(features)\n",
    "department_pred = layers.Dense(num_departments,activation='softmax',name='department')(features)\n",
    "\n",
    "model = keras.Model(inputs=[body_input,title_input,tag_input],\n",
    "                    outputs=[priority_pred,department_pred])\n",
    "model.summary()\n",
    "keras.utils.plot_model(model,'output/multi_model.png',show_shapes=True)\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n",
    "              loss={\n",
    "                  'priority':'binary_crossentropy',\n",
    "                  'department':'categorical_crossentropy'\n",
    "              },\n",
    "              loss_weights=[1.,0.2])\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "title_data = np.random.randint(num_words,size=(1280,10))\n",
    "body_data = np.random.randint(num_words,size=(1280,100))\n",
    "tag_data = np.random.randint(2,size=(1280,num_tags)).astype('float32')\n",
    "\n",
    "priority_label = np.random.random(size=(1280,1))\n",
    "department_label = np.random.randint(2,size=(1280,num_departments))\n",
    "\n",
    "history = model.fit({\n",
    "    'title':title_data,\n",
    "    'body':body_data,\n",
    "    'tag':tag_data\n",
    "},\n",
    "{'priority':priority_label,\"department\":department_label},\n",
    "batch_size=32,\n",
    "epochs=5)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n",
    "              loss={'priority':'binary_crossentropy',\n",
    "                    'department':'categorical_crossentropy'},\n",
    "              loss_weights=[1.,0.2])\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "inputs = keras.Input(shape=(32,32,3),name='img')\n",
    "h1 = layers.Conv2D(32,3,activation='relu')(inputs)\n",
    "h1 = layers.Conv2D(64,3,activation='relu')(h1)\n",
    "block1_out = layers.MaxPool2D(3)(h1)\n",
    "\n",
    "h2 = layers.Conv2D(64,3,activation='relu',padding='same')(block1_out)\n",
    "h2 = layers.Conv2D(64,3,activation='relu',padding='same')(h2)\n",
    "block2_out = layers.add([h2,block1_out])\n",
    "\n",
    "h3 = layers.Conv2D(64,3,activation='relu',padding='same')(block2_out)\n",
    "h3 = layers.Conv2D(64,3,activation='relu',padding='same')(h3)\n",
    "block3_out = layers.add([h3,block2_out])\n",
    "\n",
    "h4 = layers.Conv2D(64,3,activation='relu')(block3_out)\n",
    "h4 = layers.GlobalMaxPool2D()(h4)\n",
    "h4 = layers.Dense(256,activation='relu')(h4)\n",
    "h4 = layers.Dropout(0.5)(h4)\n",
    "\n",
    "outputs = layers.Dense(10,activation='softmax')(h4)\n",
    "\n",
    "model = keras.Model(inputs,outputs,name='small_resnet')\n",
    "model.summary()\n",
    "keras.utils.plot_model(model,'small_resnet_model.png',show_shapes=True)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "(x_train,y_train),(x_test,y_test) = keras.datasets.cifar10.load_data()\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype(\"float32\") / 255\n",
    "y_train = keras.utils.to_categorical(y_train,10)\n",
    "y_test = keras.utils.to_categorical(y_test,10)\n",
    "\n",
    "model.compile(optimizer = keras.optimizers.RMSprop(le-3),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['acc'])\n",
    "model.fit(x_train,y_train,\n",
    "          batch_size=64,\n",
    "          epochs=1,\n",
    "          validation_split=0.2)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "share_embedding = layers.Embedding(1000,64)\n",
    "input1 = keras.Input(shape=(None,),dtype='int32')\n",
    "input2 = keras.Input(shape=(None,),dtype='int32')\n",
    "\n",
    "feat1 = share_embedding(input1)\n",
    "feat2 = share_embedding(input2)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from tensorflow.keras.applications import VGG16\n",
    "vgg16 = VGG16()\n",
    "feature_list = [layer.output for layer in vgg16.layers]\n",
    "feat_ext_model = keras.Model(inputs=vgg16.input,outputs=feature_list)\n",
    "\n",
    "img = np.random.random((1,224,224,3)).astype('float32')\n",
    "ext_features = feat_ext_model(img)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class MyDense(layers.Layer):\n",
    "    def __init__(self,units=32):\n",
    "        super(MyDense,self).__init__()\n",
    "        self.units = units\n",
    "    \n",
    "    def build(self,input_shape):\n",
    "        self.w = self.add_weight(shape=(input_shape[-1],self.units),\n",
    "                                 initializer='random_normal',\n",
    "                                 trainable=True)\n",
    "        self.b = self.add_weight(shape=(self.units,),\n",
    "                                 initializer='random_normal',\n",
    "                                 trainable=True)\n",
    "    \n",
    "    def call(self,inputs):\n",
    "        return tf.matmul(inputs,self.w) + self.b\n",
    "    \n",
    "    def get_config(self):\n",
    "        return {'units':self.units}\n",
    "    \n",
    "inputs = keras.Input((4,))\n",
    "outputs = MyDense(10)(inputs)\n",
    "model = keras.Model(inputs,outputs)\n",
    "\n",
    "config = model.get_config()\n",
    "new_model = keras.Model.from_config(config,custom_objects={\"MyDense\":MyDense})\n",
    "new_model.summary()\n",
    "keras.utils.plot_model(new_model,'output/myDense.png',show_shapes=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "time_step = 10\n",
    "batch_size = 32\n",
    "hidden_dim = 32\n",
    "inputs_dim = 5\n",
    "\n",
    "class MyRnn(layers.Layer):\n",
    "    def __init__(self):\n",
    "        super(MyRnn,self).__init__()\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.projection1 = layers.Dense(units=hidden_dim,activation='tanh')\n",
    "        self.projection2 = layers.Dense(units=hidden_dim,activation='tanh')\n",
    "        self.classifier = layers.Dense(1,activation='sigmoid')\n",
    "    \n",
    "    def call(self,inputs):\n",
    "        outs = []\n",
    "        states = tf.zeros(shape=[inputs.shape[0],self.hidden_dim])\n",
    "        for t in range(inputs.shape[1]):\n",
    "            x = inputs[:,t,:]\n",
    "            h = self.projection1(x)\n",
    "            y = h + self.projection2(states)\n",
    "            states = y\n",
    "            outputs.append(y)\n",
    "        features = tf.stack(outs,axis=1)\n",
    "        print(features.shape)\n",
    "        return self.classifier(features)\n",
    "    \n",
    "inputs = keras.Input(batch_shape=(batch_size,time_step,inputs_dim))\n",
    "x = layers.Conv1D(32,3)(inputs)\n",
    "print(x.shape)\n",
    "\n",
    "outputs = MyRnn()(x)\n",
    "model = keras.Model(inputs,outputs)\n",
    "model.summary()\n",
    "keras.utils.plot_model(model,'output/myRnn.png',show_shapes=True)\n",
    "\n",
    "\n",
    "\n",
    "        \n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "rnn_model = MyRnn()\n",
    "_ = rnn_model(tf.zeros((1, 10, 5)))\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}